# -*- coding: utf-8 -*-
import requests
from lxml import etree
from bs4 import BeautifulSoup
import re
import time
from loguru import logger
import random
import traceback
import datetime
import execjs
import threading
import sys
import os
import json
from queue import Queue, Empty
from fake_useragent import UserAgent
import concurrent.futures
from src.service.requests_middler import requests_sys
from src.service.requests_middler import proxy_requests_sys
from src.service.requests_middler import insert_shipping_info
from src.service.requests_middler import read_excel
from src.service.requests_middler import read_transport_json
from src.service.requests_middler import get_proxies
current_working_directory = os.getcwd()
ua = UserAgent()
random_user_agent = ua.random
stop_threading_event = threading.Event()
ip_queue = Queue()

total_requests = 0
loss_requests = 0

def proxies_queue():
    # 获取代理并放入队列
    if ip_queue.empty():
        proxy = get_proxies()
        logger.info(f"获取ip代理成功: {proxy}")
        if proxy:
            ip_queue.put(proxy)
        else:
            logger.error(f"获取ip代理失败")
    else:
        proxy = ip_queue.get()
        return proxy

def getpoint(query):
    global total_requests
    global loss_requests
    headers = {
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://elines.coscoshipping.com/ebusiness/sailingSchedule/searchByCity",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": random_user_agent,
        "X-Client-Timestamp": "1727081372998",
        "language": "zh_CN",
        "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\"",
        "sys": "eb"
    }
    url = "https://elines.coscoshipping.com/ebbase/public/general/findCityDistrictByPrefix"
    params = {
        "prefix": query.upper(),
        "timestamp": str(time.time() * 1000)
    }
    # response = requests.get(url, headers=headers, params=params)
    # res = requests_sys(url, headers=headers, params=params, GET=True)
    retry_count = 0
    while retry_count < 3:
        proxies = proxies_queue()
        res = proxy_requests_sys(url, headers=headers, params=params, GET=True, proxies=proxies)
        if res:
            ip_queue.put(proxies)
            js_data = res.json()
            if js_data.get("code") == '200':
                return js_data['data']['content']
            else:
                logger.error(f"{query}: 未成功搜索数据")
                return None
        elif res is None:
            continue
    return None


def cosco_transport(origin_city, destination_city):
    try:
        global total_requests
        global loss_requests
        # file_path = "../maeu_ports.json"
        # cosco_query_file = "cosco_query.json"
        # port_mapping = {}
        # file_path = os.path.join(current_working_directory, file_path)
        file_path = "maeu_ports.json"
        cosco_query_file = "cosco_query.json"
        port_mapping = {}
        file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), file_path)
        cosco_query_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), cosco_query_file)
        if os.path.exists(file_path):
            with open(file_path, "r") as file:
                Menu_json = json.load(file)
            for info in Menu_json['ports']:
                port_mapping[info['portName'].upper()] = {
                    'portCode': info.get('portCode'),
                    'unLocationCode': info['unLocationCode'],
                    'portName': info['portName'],
                    'regionCode' : info.get('regionCode')
                }
        if os.path.exists(cosco_query_file):
            with open(cosco_query_file, "r") as file:
                cosco_data = json.load(file)
                cosco_data = {key.upper(): value for key, value in cosco_data.items()}
        origin_info = port_mapping.get(origin_city.upper())
        total_requests += 1
        destination_info = port_mapping.get(destination_city.upper())
        if origin_info and destination_info:

            origin_unLocationCode = origin_info['unLocationCode']
            des_unLocationCode = destination_info['unLocationCode']
            # if origin_city not in cosco_data:
            #     origin_query = getpoint(origin_city)
            #     cosco_data[origin_city] = origin_query[0] if origin_query else {}
            # else:
            #     origin_query = [cosco_data[origin_city]]
            origin_query = [cosco_data.get(origin_city.upper())]
            # if destination_city not in cosco_data:
            #     des_query = getpoint(destination_city)
            #     cosco_data[destination_city] = des_query[0] if des_query else {}
            # else:
            #     des_query = [cosco_data[destination_city]]
            des_query = [cosco_data.get(destination_city.upper())]
            if des_query:
                des_unloCode = des_query[0]['unloCode'] if des_query else ''
                with open(cosco_query_file, "w") as file:
                    json.dump(cosco_data, file, indent=4, ensure_ascii=False)
                headers = {
                    "Accept": "*/*",
                    "Accept-Language": "zh-CN,zh;q=0.9",
                    "Cache-Control": "no-cache",
                    "Connection": "keep-alive",
                    "Content-Type": "application/json",
                    "Origin": "https://elines.coscoshipping.com",
                    "Pragma": "no-cache",
                    "Referer": "https://elines.coscoshipping.com/ebusiness/sailingSchedule/searchByCity/resultByCity",
                    "Sec-Fetch-Dest": "empty",
                    "Sec-Fetch-Mode": "cors",
                    "Sec-Fetch-Site": "same-origin",
                    "User-Agent": random_user_agent,
                    "X-Client-Timestamp": str(time.time() * 1000),
                    "language": "zh_CN",
                    "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
                    "sec-ch-ua-mobile": "?0",
                    "sec-ch-ua-platform": "\"Windows\"",
                    "sys": "eb"
                }
                url = "https://elines.coscoshipping.com/ebschedule/public/purpoShipmentWs"
                # 出发时间加27天 /
                date = datetime.datetime.now()
                toDate = (date + datetime.timedelta(days=27)).strftime('%Y-%m-%d')
                date = date.strftime('%Y-%m-%d') # 重新格式化出发时间
                data = {
                    "fromDate": f"{date}", # f"{date}-01"
                    "pickup": "B",
                    "delivery": "B",
                    "estimateDate": "D",
                    "toDate": f"{toDate}", # f"{date}-30"
                    "originCityUuid": origin_query[0]['cityUuid'],
                    "destinationCityUuid": des_query[0]['cityUuid'],
                    "originCity": origin_query[0]['fullFormate'] + ',' + origin_query[0]['unloCode'],
                    "destinationCity": des_query[0]['fullFormate'] + ',' + des_unloCode,
                    "cargoNature": "All"
                }
                data = json.dumps(data, separators=(',', ':'))
                retry_count = 0
                while retry_count < 3:
                    proxies = proxies_queue()
                    res = proxy_requests_sys(url, headers=headers, data=data, POST=True, proxies=proxies)
                    # res = requests_sys(url, headers=headers, data=data, POST=True)
                    if res:
                        ip_queue.put(proxies)
                        js_data = res.json()
                        logger.info(f"当前查询 起点:{origin_city} ----> 终点: {destination_city}")
                        logger.info(f"当前请求总数: {total_requests}, 失败请求总数: {loss_requests}")
                        # 终点站代码
                        terminal_station_code = f"{destination_city}({port_mapping.get(destination_city.upper())['unLocationCode']})"
                        # tretim_code = [cosco_data[destination_city.upper()]][0]['stateCode']
                        # tmp_list = []
                        if js_data['data'].get("content").get("data"):
                            index = 0
                            while index < len(js_data['data']['content']['data']):
                                next_info = None
                                ###################
                                item = {}
                                item['origin_code'] = f"{origin_city}({origin_query[0]['unloCode']})"
                                info = js_data['data']['content']['data'][index]
                                item['shipping_lanes'] = js_data['data']['content']['data'][index]['service']
                                pod = info['pod']
                                # des_unloCode = [cosco_data[pod.upper()]][0]['unloCode']
                                des_unloCode = port_mapping.get(pod.upper())['unLocationCode']
                                if info['available'] is not None:
                                    # 存在时间
                                    item['destination_code'] = f"{info['pod']}({des_unloCode})"
                                    # info['eta']
                                    item['arrive_date'] = datetime.datetime.strptime(info['available'], "%Y-%m-%d %H:%M").strftime("%Y-%m-%d")
                                else:
                                    if index + 1 < len(js_data['data']['content']['data']):
                                        next_info = js_data['data']['content']['data'][index + 1]
                                        pod = next_info['pod']
                                        des_unloCode = [cosco_data[pod.upper()]][0]['unloCode']
                                        item['destination_code'] = f"{next_info['pod']}({des_unloCode})"
                                        # shanghai ---> Buffalo
                                        item['arrive_date'] = datetime.datetime.strptime(next_info['available'], "%Y-%m-%d %H:%M").strftime(
                                            "%Y-%m-%d")
                                        index += 1
                                    else:
                                        print("跳过")
                                        continue
                                item['transport_contractor'] = "COSCO"
                                if next_info:
                                    # 出发时间修改
                                    item['depart_date'] = datetime.datetime.strptime(next_info['etd'], "%Y-%m-%d %H:%M").strftime("%Y-%m-%d")
                                    item['ship_name'] = next_info["vessel"]
                                    item['voyage_number'] = next_info["extVoyage"]
                                else:
                                    item['depart_date'] = datetime.datetime.strptime(info['etd'], "%Y-%m-%d %H:%M").strftime("%Y-%m-%d")
                                    item['ship_name'] = info["vessel"]
                                    item['voyage_number'] = info["extVoyage"]
                                item['haulage_time'] = info["transitTime"]

                                # 判断获取的是否为目的地
                                # 增加中转站数据字段
                                regionCode = port_mapping.get(destination_city.upper())['regionCode']
                                if item['destination_code'].upper() != terminal_station_code.upper():
                                    item['transfer_code'] = item['destination_code']
                                    if regionCode is None:
                                        # 如果州代码为空就
                                        item['destination_code'] = f"{destination_city}({regionCode})"
                                    else:
                                        item['destination_code'] = f"{destination_city},{regionCode}({port_mapping.get(destination_city.upper())['unLocationCode']})"
                                else:
                                    destination_code_one, destination_code_two = item['destination_code'].split('(')
                                    item['destination_code'] = f"{destination_code_one},{[cosco_data[destination_code_one.upper()]][0]['stateCode']}({destination_code_two}"
                                    item['transfer_code'] = None
                                # tmp_list.append(item)
                                insert_shipping_info(item)
                                # print(item)
                                index += 1

                        else:
                            logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据")
                            read_transport_json(transport='COSCO', origin_city=origin_city, des_city=destination_city,
                                           fail_result="未查询到数据")
                            loss_requests += 1
                            retry_count += 1

                        break
                    elif res == 0:
                        retry_count += 1
                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据, 请求超时")

                if retry_count == 3:
                    read_transport_json(transport='COSCO', origin_city=origin_city, des_city=destination_city,
                                        fail_result="请求超时")
                    loss_requests += 1

            else:
                read_transport_json(transport='COSCO', origin_city=origin_city, des_city=destination_city,
                               fail_result="未查询到数据")
                logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 未查询到数据")
                loss_requests += 1

    except Exception as e:
        logger.error(f"{origin_city} ----> {destination_city}: {e}")
        print(traceback.format_exc())


def process_route(origin_city, destination_list):
    for destination_city in destination_list:
        cosco_transport(origin_city, destination_city)

def cosco_spider_run():
    global total_requests
    global loss_requests
    # origin_city_list = [
    #     "Shanghai", "XIAMEN", "NANSHA", "QINGDAO", "YANTIAN", "NINGBO"
    # ]
    #
    # destination_list = [
    #     "Los Angeles", "Long Beach", "Atlanta", "Baltimore", "Boston", "Buffalo", "Calgary", "Charleston",
    #     "Chicago", "Chippewa Falls", "Cincinnati", "Cleveland", "Dallas", "Denver",
    #     "Detroit", "Edmonton", "EL Paso", "Halifax", "Houston", "Jacksonville",
    #     "Kansas City", "Louisville", "Memphis", "Miami", "Minneapolis", "Montreal",
    #     "Nashville", "New Orleans", "New York", "Norfolk", "Oakland", "Omaha",
    #     "Philadelphia", "Pittsburgh", "Portland", "Prince Rupert", "Saint Louis",
    #     "Salt Lake City", "San Antonio", "Saskatoon", "Savannah", "Seattle",
    #     "Tacoma", "Tampa", "Toronto", "Vancouver", "Winnipeg"
    # ]

    json_path_one = r"origin_and_destination.json"
    # json_path_one = r"origin_and_destination - 副本.json"
    json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), json_path_one)
    with open(f'{json_path}', 'r', encoding='utf-8') as f:
        tmp = json.loads(f.read())

    origin_city_list = tmp['origin_city_list']
    destination_list = tmp['destination_list']
    del tmp, json_path

    # with concurrent.futures.ThreadPoolExecutor() as executor:
    #     futures = [executor.submit(process_route, origin_city, destination_list) for origin_city in origin_city_list]

    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = []
        for origin_city in origin_city_list:
            future = executor.submit(process_route, origin_city, destination_list)
            futures.append(future)
            time.sleep(1)

        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except Exception as e:
                # 输出异常详细信息，包括出错的行号
                print(f"An error occurred: {e}")
                print(traceback.format_exc())
    read_excel(transport="COSCO", total_requests=total_requests, loss_requests=loss_requests)

if __name__ == '__main__':
    cosco_spider_run()